package net.trustie.webmagic.one;

import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;

import javax.annotation.Resource;

import net.trustie.webmagic.extrator.CSSPathExtractor;
import net.trustie.webmagic.extrator.Extractor;
import net.trustie.webmagic.extrator.URLPatternExtractor;
import net.trustie.webmagic.one.dao.DetailHtmlDao;
import net.trustie.webmagic.one.dao.ErrorPageDao;
import net.trustie.webmagic.one.dao.URLDao;
import net.trustie.webmagic.one.model.DetailHtml;
import net.trustie.webmagic.utils.LongSleepHelper;
import net.trustie.webmagic.utils.MyRandomer;
import net.trustie.webmagic.utils.SubSite;

import org.apache.log4j.Logger;
import org.springframework.context.ApplicationContext;
import org.springframework.context.support.ClassPathXmlApplicationContext;
import org.springframework.stereotype.Component;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Request;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Task;
import us.codecraft.webmagic.downloader.HttpClientDownloader;
import us.codecraft.webmagic.selector.Html;
import us.codecraft.webmagic.utils.UrlUtils;

@Component
public class MakeUp {
	private HttpClientDownloader downloader = new HttpClientDownloader();
	private Configure conf;
	private Set<Integer> acceptStatCode;
	private String domain;
	private Site site;
	private MyTask2 task = new MyTask2();
	Logger logger = Logger.getLogger(this.getClass());
	Map<String,Set<String>> lastPages = new HashMap<String,Set<String>>();
	
	@Resource
	private DetailHtmlDao detailHtmlDao;

	@Resource
	private URLDao urlDao;

	@Resource
	private ErrorPageDao errorPageDao;
	private String detailHtmlTableName;
	private String errorPageTableName;
	private String urlTableName;
	SimpleDateFormat extractedTimeFormat = new SimpleDateFormat(
			"yyyy-MM-dd HH:mm:ss");
	public static void main(String args[]){
		String configureName = "";
		if (args.length != 0)
			configureName = args[0].toString();
		if (configureName.equals("")) {
			configureName = "ossean";
		} else {
		}
		ApplicationContext applicationContext = new ClassPathXmlApplicationContext(
				"classpath:/spring/applicationContext*.xml");
		final MakeUp htmlCrawler = applicationContext
				.getBean(MakeUp.class);
		htmlCrawler.crawl(configureName);
		System.out.println("this is in th crawler");
	}

	
	private void crawl(String configureName) {
		// TODO Auto-generated method stub
		init(configureName);
		
		Extractor extractor = null;
		if ("URLPattern".equals(conf.getExtractMethod())) {
			extractor = new URLPatternExtractor(conf.getUrlPost());
		} else if ("CSSPath".equals(conf.getExtractMethod())) {
			extractor = new CSSPathExtractor(conf.getUrlPost());
		} else {
			logger.error("extract method error!!!");
		}
		
		int subSiteNum = this.conf.getSubSiteNum();
		logger.info("该站点共有" + subSiteNum + "个子站点");	
		
		SubSite subSite;
		
	
		int doneNum = 0;//记录已完成镜像爬取的子站点个数
		while(doneNum ++< subSiteNum){
			//conf 负责获得这次处理的站点
			subSite = this.conf.nexSite();
			
			logger.info("现在爬取子站点  " + subSite.getName());
			int start = subSite.getStartPageIndex();
			int end = subSite.getEndInPageIndex();
			int j =0;
			int i =0;
			for( i = start;i <= end; i ++,j++){
				Page page = downloader.download(new Request(
						subSite.getPrefixUrl()
						+ i
						+ subSite.getPostfixUrl()), task);
				
				Html html = new Html(UrlUtils.fixAllRelativeHrefs(
						page.getRawText(),this.conf.getFixAllRelativeHrefs()));

				Set<String> urls = extractor.extract(html);
				
				for(String url : urls){
					Date date = new Date();
					String extractedTime = extractedTimeFormat.format(date);
					urlDao.addURL(urlTableName, url, extractedTime);
				}
				try {
					Thread.sleep(MyRandomer.getRandomNum(15, 30) * 1000);
				} catch (InterruptedException e) {
					// TODO Auto-generated catch block
					e.printStackTrace();
				}
			}
			
		}
		
	}


	private void init(String configureName) {
		// TODO Auto-generated method stub
		this.conf = new Configure(configureName);
		this.acceptStatCode = this.initAcceptStatCode();
		this.domain = conf.getDomain();
		this.site = Site.me().setSleepTime(conf.getSleepTimeThread())
				.setTimeOut(conf.getTimeOut())
				.setRetryTimes(conf.getRetryTimes())
				.setCycleRetryTimes(conf.getCycleRetryTimes())
				.setDomain(conf.getDomain()).setUserAgent(conf.getUserAgent())
				.setAcceptStatCode(acceptStatCode);
		task.site = site;
		
		this.detailHtmlTableName = this.domain + "_html_detail";
		this.errorPageTableName = this.domain + "_error_page";
		this.urlTableName = this.domain + "_url";
	}


	private Set<Integer> initAcceptStatCode() {
		Set<Integer> acceptStatCode = new HashSet<Integer>();
		acceptStatCode.add(200);
		acceptStatCode.add(404);
		acceptStatCode.add(500);
		acceptStatCode.add(503);
		return acceptStatCode;
	}
}

class MyTask2 implements Task{
	Site site;
	@Override
	public String getUUID() {
		// TODO Auto-generated method stub
		return null;
	}

	@Override
	public Site getSite() {
		// TODO Auto-generated method stub
		return site;
	}

	@Override
	public int getThreadAlive() {
		// TODO Auto-generated method stub
		return 0;
	}
}
